# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
import json
import logging
import os.path
import platform

from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem

from ouyeelMall.api.addnews import AddNews
from ouyeelMall.constant import ProjectPath
from utils.hash_funs import my_hash


class OuyeelmallPipeline:
    def __init__(self):
        self.newsAdder = AddNews()

    def process_item(self, item: dict, spider):
        thumb_url = item['post[thumb]']
        thumb = self.newsAdder.upload_pic(remote_url=thumb_url)
        item["post[thumb]"] = thumb
        try:
            # with open(f"datas/{item['post[title]']}.json", 'w', encoding='U8') as fp:
            #     json.dump(item, fp, ensure_ascii=False)
            resp = self.newsAdder.add(payload=item)
        except Exception as e:
            logging.warning(f"Save item failed for item: {item}; reason is {e.args}")
        finally:
            return item


class OuyeelmallDuplicatePipeline:
    def __init__(self):
        if "windows" in platform.platform().lower():
            filepath = os.path.join(r"C:\Users\YW\gerapy\projects\ouyeelMall", "doc", "duplicate_items.txt")
        else:
            filepath = os.path.join(ProjectPath, "doc", "duplicate_items.txt")
        self.dup_file = filepath
        self.item_set = set()

    def open_spider(self, spider):
        with open(self.dup_file, 'r') as fp:
            lines = fp.readlines()
        self.item_set.update(l.strip() for l in lines)

    def close_spider(self, spider):
        self.item_set.clear()

    def process_item(self, item, spider):
        title = item['post[title]']
        ht = my_hash(title)
        if ht in self.item_set:
            raise DropItem(f"Duplicate title: {title}")

        self.item_set.add(ht)
        with open(self.dup_file, 'a+') as fp:
            fp.write(ht)
            fp.write("\n")

        return item
